{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import init, gluon, nd, autograd, image\n",
    "from mxnet.gluon import nn\n",
    "from mxnet.gluon.data import vision\n",
    "from mxnet.gluon.model_zoo import vision as models\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import cv2\n",
    "import h5py\n",
    "import os\n",
    "from glob import glob\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "ctx = mx.gpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    net = nn.Sequential()\n",
    "    with net.name_scope():\n",
    "        net.add(nn.BatchNorm())\n",
    "        net.add(nn.Dense(1024))\n",
    "        net.add(nn.BatchNorm())\n",
    "        net.add(nn.Activation('relu'))\n",
    "        net.add(nn.Dropout(0.5))\n",
    "        net.add(nn.Dense(512))\n",
    "        net.add(nn.BatchNorm())\n",
    "        net.add(nn.Activation('relu'))\n",
    "        net.add(nn.Dropout(0.5))\n",
    "        net.add(nn.Dense(120))\n",
    "    net.initialize(ctx=ctx)\n",
    "    return net\n",
    "\n",
    "def accuracy(output, labels):\n",
    "    return nd.mean(nd.argmax(output, axis=1) == labels).asscalar()\n",
    "\n",
    "def evaluate(net, data_iter):\n",
    "    loss, acc, n = 0., 0., 0.\n",
    "    steps = len(data_iter)\n",
    "    for data, label in data_iter:\n",
    "        data, label = data.as_in_context(ctx), label.as_in_context(ctx)\n",
    "        output = net(data)\n",
    "        acc += accuracy(output, label)\n",
    "        loss += nd.mean(softmax_cross_entropy(output, label)).asscalar()\n",
    "    return loss/steps, acc/steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20580/20580 [00:01<00:00, 18391.75it/s]\n"
     ]
    }
   ],
   "source": [
    "synset = list(pd.read_csv('sample_submission.csv').columns[1:])\n",
    "n = len(glob('Images/*/*.jpg'))\n",
    "\n",
    "y = nd.zeros((n,))\n",
    "for i, file_name in tqdm(enumerate(glob('Images/*/*.jpg')), total=n):\n",
    "    y[i] = synset.index(file_name.split('/')[1][10:].lower())\n",
    "    nd.waitall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = [nd.load('features_incep.nd')[0], nd.load('features_res.nd')[0]]\n",
    "features = nd.concat(*features, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_iter_train = gluon.data.DataLoader(gluon.data.ArrayDataset(features, y), 128, shuffle=True)\n",
    "\n",
    "softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "net = build_model()\n",
    "trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': 1e-3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. loss: 1.1068, acc: 76.33%\n",
      "Epoch 10. loss: 0.1229, acc: 95.91%\n",
      "Epoch 20. loss: 0.0781, acc: 97.35%\n",
      "Epoch 30. loss: 0.0525, acc: 98.30%\n",
      "Epoch 40. loss: 0.0452, acc: 98.50%\n",
      "Epoch 50. loss: 0.0286, acc: 99.02%\n",
      "Epoch 60. loss: 0.0177, acc: 99.45%\n",
      "Epoch 70. loss: 0.0127, acc: 99.53%\n",
      "Epoch 80. loss: 0.0119, acc: 99.64%\n",
      "Epoch 90. loss: 0.0101, acc: 99.66%\n",
      "Epoch 99. loss: 0.0094, acc: 99.67%\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.002932575075505811, 0.99820458074534157)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    if epoch == 50:\n",
    "        trainer.set_learning_rate(1e-4)\n",
    "    if epoch == 80:\n",
    "        trainer.set_learning_rate(1e-5)\n",
    "    train_loss = 0.\n",
    "    train_acc = 0.\n",
    "    steps = len(data_iter_train)\n",
    "    for data, label in data_iter_train:\n",
    "        data, label = data.as_in_context(ctx), label.as_in_context(ctx)\n",
    "        with autograd.record():\n",
    "            output = net(data)\n",
    "            loss = softmax_cross_entropy(output, label)\n",
    "        loss.backward()\n",
    "        trainer.step(batch_size)\n",
    "        train_loss += nd.mean(loss).asscalar()\n",
    "        train_acc += accuracy(output, label)\n",
    "    if epoch%10 == 0:\n",
    "        print(\"Epoch %d. loss: %.4f, acc: %.2f%%\" % (epoch, train_loss/steps, train_acc/steps*100))\n",
    "print(\"Epoch %d. loss: %.4f, acc: %.2f%%\" % (epoch, train_loss/steps, train_acc/steps*100))\n",
    "evaluate(net, data_iter_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_names = ['inceptionv3', 'resnet152_v1']\n",
    "features_test = [nd.load('features_test_%s.nd' % model_name)[0] for model_name in model_names]\n",
    "features_test = nd.concat(*features_test, dim=1)\n",
    "\n",
    "output = nd.softmax(net(features_test.as_in_context(ctx))).asnumpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_pred = pd.read_csv('sample_submission.csv')\n",
    "\n",
    "for i, c in enumerate(df_pred.columns[1:]):\n",
    "    df_pred[c] = output[:,i]\n",
    "\n",
    "df_pred.to_csv('pred.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
